const puppeteer = require('puppeteer');
const ora = require('ora');
const FakeUseragent = require('fake-useragent');
const fs = require('fs');
const Paint = require('../tool/console.color');


let targetWebsiteURL = 'https://zz.lianjia.com/ershoufang/';
let pageOption = {
  timeout: 0,
  waitUntil: 'domcontentloaded'
}

// 文件是否存在
function fileIsExist(filePath) {
  try {
    fs.readFileSync(filePath);
    return true;
  } catch {
    return false;
  }
}

async function getData(fileName) {
  // 创建浏览器窗口
  Paint('新建浏览器中...', -1);
  const browser = await puppeteer.launch({
    headless: true
  })

  // 创建标签页
  Paint('新建页面中...', -1);
  const page = await browser.newPage();

  // 进入待爬取页面
  await page.goto(targetWebsiteURL);

  let arealist = await page.$$eval('div[data-role=ershoufang]>div:nth-child(1)>a', els => els.map(item => ({
    name: item.innerHTML,
    url: item.href
  })));

  let allStreetNameList = [];
  let processLoading = ora('0%').start();

  for (let i in arealist) {
    let other_page = await browser.newPage();
    other_page.setUserAgent(FakeUseragent());
    await other_page.goto(arealist[i].url, pageOption)
    let streetlist = await other_page.$$eval('div[data-role=ershoufang]>div:nth-child(2)>a', els => els.map(item => ({
      name: item.innerHTML,
      url: item.href
    })))
    let listAfterFilter = streetlist.filter(item => !allStreetNameList.find(val => item.name === val.name));
    arealist[i].street = {}
    listAfterFilter.forEach(item => {
      arealist[i].street[item.name] = item.url;
    })
    allStreetNameList = [...allStreetNameList, ...listAfterFilter];
    // * 1. 用自定义的颜色打印函数
    // Paint(`${+i + 1}/${arealist.length} ${arealist[i].name} 记录完毕！`, -1);

    // * 2. 使用ora显示进度
    processLoading.text = parseInt((+i + 1) / arealist.length * 100) + '%';
  }

  fs.writeFile(fileName, JSON.stringify(allStreetNameList), err => {
    if (err) {
      console.error(err);
    } else {
      // * 1. 用自定义的颜色打印函数
      // Paint('街道数据收集成功！');

      // * 2. 使用ora显示进度
      processLoading.succeed('街道数据收集成功！');
    }
  })

  await browser.close();
}

let fileName = 'data.json'

getData(fileName);
